Skip to content

Commit e6464ef

Browse files
smithellissmithellis
committed
Replace apscheduler with celery beat
1 parent 5b4817b commit e6464ef

File tree

10 files changed

+409
-1073
lines changed

10 files changed

+409
-1073
lines changed

‎docker-compose.yml

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ services:
1616
- kibana
1717
- redis
1818
- celery
19+
- celery-beat
1920
- mailcatcher
2021
ports:
2122
- "8000:8000"
@@ -68,6 +69,20 @@ services:
6869
- postgres
6970
- redis
7071

72+
celery-beat:
73+
build:
74+
context: .
75+
target: base
76+
command: celery -A kitsune beat -l info
77+
env_file: .env
78+
volumes:
79+
- ./:/app:delegated
80+
user: ${UID:-kitsune}
81+
depends_on:
82+
- postgres
83+
- redis
84+
- celery
85+
7186
mailcatcher:
7287
image: schickling/mailcatcher
7388
ports:

‎kitsune/celery.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
app = Celery("kitsune")
99
app.config_from_object("django.conf:settings", namespace="CELERY")
1010
app.autodiscover_tasks()
11+
1112
app.conf.task_routes = {
1213
"post_office.tasks.send_queued_mail": {"queue": "email"},
1314
"kitsune.tidings.tasks.send_emails": {"queue": "email"},
@@ -17,3 +18,5 @@
1718
"kitsune.wiki.tasks.send_reviewed_notification": {"queue": "email"},
1819
"kitsune.wiki.tasks.send_contributor_notification": {"queue": "email"},
1920
}
21+
22+
app.conf.beat_schedule = settings.CELERY_BEAT_SCHEDULE

‎kitsune/celery_schedules.py

Lines changed: 157 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,157 @@
1+
from celery.schedules import crontab
2+
3+
from kitsune.tasks import (
4+
auto_archive_old_questions,
5+
cache_most_unhelpful_kb_articles,
6+
cleanup_expired_users,
7+
cleanup_mail,
8+
cleanup_old_account_events,
9+
cohort_analysis,
10+
enqueue_lag_monitor_task,
11+
fix_current_revisions,
12+
generate_missing_share_links,
13+
rebuild_kb,
14+
reload_question_traffic_stats,
15+
reload_wiki_traffic_stats,
16+
report_employee_answers,
17+
reindex_es,
18+
send_queued_mail,
19+
send_welcome_emails,
20+
send_weekly_ready_for_review_digest,
21+
update_contributor_metrics,
22+
update_l10n_coverage_metrics,
23+
update_l10n_contributor_metrics,
24+
update_l10n_metric,
25+
update_product_details,
26+
update_top_contributors,
27+
update_weekly_votes,
28+
)
29+
30+
# Every 10 minutes
31+
CELERY_BEAT_SCHEDULE = {
32+
"enqueue-lag-monitor-task": {
33+
"task": enqueue_lag_monitor_task.name,
34+
"schedule": crontab(minute="*/10"),
35+
},
36+
# Every 15 minutes
37+
"send-queued-mail": {
38+
"task": send_queued_mail.name,
39+
"schedule": crontab(minute="*/15"),
40+
},
41+
# Every hour at minute 30
42+
"send-welcome-emails": {
43+
"task": send_welcome_emails.name,
44+
"schedule": crontab(minute="30"),
45+
},
46+
# Every hour at minute 45
47+
"reindex-es": {
48+
"task": reindex_es.name,
49+
"schedule": crontab(minute="45"),
50+
},
51+
# Every 6 hours at minute 00
52+
"update-product-details": {
53+
"task": update_product_details.name,
54+
"schedule": crontab(hour="*/6", minute="00"),
55+
},
56+
# Every 6 hours at minute 20
57+
"generate-missing-share-links": {
58+
"task": generate_missing_share_links.name,
59+
"schedule": crontab(hour="*/6", minute="20"),
60+
},
61+
# Every Sunday at 12:00
62+
"cleanup-mail": {
63+
"task": cleanup_mail.name,
64+
"schedule": crontab(hour="12", minute="0", day_of_week="0"),
65+
},
66+
# Every Sunday at 00:00
67+
"rebuild-kb": {
68+
"task": rebuild_kb.name,
69+
"schedule": crontab(hour="00", minute="00", day_of_week="0"),
70+
},
71+
# Every Sunday at 00:42
72+
"update-top-contributors": {
73+
"task": update_top_contributors.name,
74+
"schedule": crontab(hour="00", minute="42", day_of_week="0"),
75+
},
76+
# Every Sunday at 01:00
77+
"update-l10n-coverage-metrics": {
78+
"task": update_l10n_coverage_metrics.name,
79+
"schedule": crontab(hour="01", minute="00", day_of_week="0"),
80+
},
81+
# Every Sunday at 01:11
82+
"report-employee-answers": {
83+
"task": report_employee_answers.name,
84+
"schedule": crontab(hour="01", minute="11", day_of_week="0"),
85+
},
86+
# Every Sunday at 01:40
87+
"update-weekly-votes": {
88+
"task": update_weekly_votes.name,
89+
"schedule": crontab(hour="01", minute="40", day_of_week="0"),
90+
},
91+
# Every Sunday at 03:00
92+
"update-contributor-metrics": {
93+
"task": update_contributor_metrics.name,
94+
"schedule": crontab(hour="03", minute="00", day_of_week="0"),
95+
},
96+
# Every day at 04:00
97+
"auto-archive-old-questions": {
98+
"task": auto_archive_old_questions.name,
99+
"schedule": crontab(hour="04", minute="00"),
100+
},
101+
# Every day at 10:00
102+
"update-l10n-metric": {
103+
"task": update_l10n_metric.name,
104+
"schedule": crontab(hour="10", minute="00"),
105+
},
106+
# Every day at 16:00
107+
"reload-wiki-traffic-stats": {
108+
"task": reload_wiki_traffic_stats.name,
109+
"schedule": crontab(hour="16", minute="00"),
110+
},
111+
# Every day at 21:00
112+
"cache-most-unhelpful-kb-articles": {
113+
"task": cache_most_unhelpful_kb_articles.name,
114+
"schedule": crontab(hour="21", minute="00"),
115+
},
116+
# Every day at 23:00
117+
"reload-question-traffic-stats": {
118+
"task": reload_question_traffic_stats.name,
119+
"schedule": crontab(hour="23", minute="00"),
120+
},
121+
# Every Friday at 04:00
122+
"send-weekly-ready-for-review-digest": {
123+
"task": send_weekly_ready_for_review_digest.name,
124+
"schedule": crontab(hour="04", minute="00", day_of_week="5"),
125+
},
126+
# Every Monday at 00:30
127+
"cohort-analysis": {
128+
"task": cohort_analysis.name,
129+
"schedule": crontab(hour="00", minute="30", day_of_week="1"),
130+
},
131+
# First day of every month at 00:30
132+
"update-l10n-contributor-metrics": {
133+
"task": update_l10n_contributor_metrics.name,
134+
"schedule": crontab(hour="00", minute="30", day_of_month="1"),
135+
},
136+
# Every Sunday at 02:00
137+
"cleanup-old-account-events": {
138+
"task": cleanup_old_account_events.name,
139+
"schedule": crontab(hour="02", minute="00", day_of_week="0"),
140+
},
141+
# Every Sunday at 03:00
142+
"cleanup-expired-users": {
143+
"task": cleanup_expired_users.name,
144+
"schedule": crontab(hour="03", minute="00", day_of_week="0"),
145+
},
146+
# Every Sunday at 00:00
147+
"fix-current-revisions": {
148+
"task": fix_current_revisions.name,
149+
"schedule": crontab(hour="00", minute="00", day_of_week="0"),
150+
},
151+
# Every 4 hours, 15 minutes after the hour
152+
"process-unprocessed-account-events": {
153+
"task": "kitsune.users.tasks.process_unprocessed_account_events",
154+
"schedule": crontab(minute="15", hour="*/4"),
155+
"args": [1],
156+
},
157+
}

0 commit comments

Comments
 (0)